The goals / steps of this project are the following:
import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
%matplotlib inline
#matplotlib qt
def show_images(images):
num_img = len(images)
#print(num_img)
#plt.figure(figsize=(15,5))
#fig, axes = plt.subplots(num_img, 1)
#for axe, i in zip(axes.flat, range(num_img)):
# print(i)
# axe.imshow(images[i])
# #plt.imshow(images[i])
for i in range(num_img):
#print("i:",i)
plt.figure(i)
#plt.subplot(num_img, 1, i+1)
plt.imshow(images[i])
#plt.imshow(images[i])
#plt.show()
def show_two_images(img1, img2, title1="Original Image", title2="Result Image"):
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img1)
ax1.set_title(title1, fontsize=50)
ax2.imshow(img2)
ax2.set_title(title2, fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
def show_pair_images(images1, images2):
for i in range(len(images1)):
show_two_images(images1[i], imges2[i])
First, I'll compute the camera calibration using chessboard images
# a function that takes an image, object points, and image points
# performs the camera calibration, image distortion correction and
# returns the undistorted image
def cal_undistort(img, objpoints, imgpoints):
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
img_dst = cv2.undistort(img, mtx, dist, None, mtx)
return img_dst
def compute_camera_calibration(images,chessboard_size_x=9, chessboard_size_y=6):
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((chessboard_size_y*chessboard_size_x,3), np.float32)
objp[:,:2] = np.mgrid[0:chessboard_size_x,0:chessboard_size_y].T.reshape(-1,2)
# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.
# Make a list of calibration images
images_converted = []
# Step through the list and search for chessboard corners
for fname in images:
#print(fname)
img = cv2.imread(fname)
img_converted = np.copy(img)
gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
# Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, (chessboard_size_x,chessboard_size_y),None)
# If found, add object points, image points
if ret == True:
objpoints.append(objp)
imgpoints.append(corners)
# Draw and display the corners
img_converted = cv2.drawChessboardCorners(img_converted, (chessboard_size_x,chessboard_size_y), corners, ret)
images_converted.append(img_converted)
# Undistort the original image using objpoints and imgpoints
img_undistorted = cal_undistort(img, objpoints, imgpoints)
show_two_images(img_converted, img_undistorted)
return objpoints, imgpoints
camera_images = glob.glob('./camera_cal/calibration*.jpg')
objpoints, imgpoints = compute_camera_calibration(camera_images)
There are two main steps to this process: use chessboard images to obtain image points and object points, and then use the OpenCV functions cv2.calibrateCamera() and cv2.undistort() to compute the calibration and undistortion.
Try computing the calibration and undistortion in the exercise below, and if you want to play with extracting object points and image points yourself, fork the Jupyter notebook and images in this repository.
If you run into any errors as you run your code, please refer to the Examples of Useful Code section in the previous video and make sure that your code syntax matches up!
test_img = cv2.imread('./camera_cal/calibration2.jpg')
undistorted = cal_undistort(test_img, objpoints, imgpoints)
show_two_images(test_img, undistorted, title2="Undistorted Image")
Compute the perspective transform, M, given source and destination points:
##M = cv2.getPerspectiveTransform(src, dst)
Compute the inverse perspective transform:
##Minv = cv2.getPerspectiveTransform(dst, src)
Warp an image using the perspective transform, M:
##warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
Note: When you apply a perspective transform, choosing four source points manually, as we did in this video, is often not the best option. There are many other ways to select source points. For example, many perspective transform algorithms will programmatically detect four source points in an image based on edge or corner detection and analyzing attributes like color and surrounding pixels.
Here's a tricky quiz for you! You have now seen how to find corners, calibrate your camera, undistort an image, and apply a perspective transform. Now it's your chance to perform all these steps on an image. In the last quiz you calibrated the camera, so here I'm giving you the camera matrix, mtx, and the distortion coefficients dist to start with.
Your goal is to generate output like the image shown above. To do that, you need to write a function that takes your distorted image as input and completes the following steps:
Undistort the image using cv2.undistort() with mtx and dist Convert to grayscale Find the chessboard corners Draw corners Define 4 source points (the outer 4 corners detected in the chessboard pattern) Define 4 destination points (must be listed in the same order as src points!) Use cv2.getPerspectiveTransform() to get M, the transform matrix use cv2.warpPerspective() to apply M and warp your image to a top-down view HINT: Source points are the x and y pixel values of any four corners on your chessboard, you can extract these from the corners array output from cv2.findChessboardCorners(). Your destination points are the x and y pixel values of where you want those four corners to be mapped to in the output image. If you run into any errors as you run your code, please refer to the Examples of Useful Code section in the previous video and make sure that your code syntax matches up! For this example, please also refer back to the examples in the Calibrating Your Camera video.
"""
import pickle
import cv2
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# Read in the saved camera matrix and distortion coefficients
# These are the arrays you calculated using cv2.calibrateCamera()
#dist_pickle = pickle.load( open( "wide_dist_pickle.p", "rb" ) )
#mtx = dist_pickle["mtx"]
#dist = dist_pickle["dist"]
# Read in an image
img = cv2.imread('test_image2.png')
nx = 8 # the number of inside corners in x
ny = 6 # the number of inside corners in y
# MODIFY THIS FUNCTION TO GENERATE OUTPUT
# THAT LOOKS LIKE THE IMAGE ABOVE
def corners_unwarp(img, nx, ny, mtx, dist):
# Pass in your image into this function
# Write code to do the following steps
# 1) Undistort using mtx and dist
img_dst = cv2.undistort(img, mtx, dist, None, mtx)
# 2) Convert to grayscale
gray = cv2.cvtColor(img_dst,cv2.COLOR_BGR2GRAY)
# 3) Find the chessboard corners
ret, corners = cv2.findChessboardCorners(gray, (8,6),None)
print(ret)
print(len(corners))
print(corners)
# 4) If corners found:
warped = np.copy(gray)
M = None
if ret == True:
# a) draw corners
img_w_corners = cv2.drawChessboardCorners(img_dst, (8,6), corners, ret)
# Choose offset from image corners to plot detected corners
# This should be chosen to present the result at the proper aspect ratio
# My choice of 100 pixels is not exact, but close enough for our purpose here
offset = 100 # offset for dst points
# Grab the image shape
img_size = (gray.shape[1], gray.shape[0])
# b) define 4 source points src = np.float32([[,],[,],[,],[,]])
#Note: you could pick any four of the detected corners
# as long as those four corners define a rectangle
#One especially smart way to do this would be to use four well-chosen
# corners that were automatically detected during the undistortion steps
#We recommend using the automatic detection of corners in your code
src = np.float32([corners[0], corners[nx-1], corners[-1], corners[-nx]])
#src = np.float32([corners[0],corners[7],corners[40],corners[47]])#([[,],[,],[,],[,]])
# c) define 4 destination points dst = np.float32([[,],[,],[,],[,]])
# For destination points, I'm arbitrarily choosing some points to be
# a nice fit for displaying our warped result
# again, not exact, but close enough for our purposes
dst = np.float32([[offset, offset], [img_size[0]-offset, offset],
[img_size[0]-offset, img_size[1]-offset],
[offset, img_size[1]-offset]])
#dst = np.float32([[,],[,],[,],[,]])
# d) use cv2.getPerspectiveTransform() to get M, the transform matrix
M = cv2.getPerspectiveTransform(src, dst)
# e) use cv2.warpPerspective() to warp your image to a top-down view
warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
#warped = img_w_corners
#delete the next two lines
return warped, M
top_down, perspective_M = corners_unwarp(img, nx, ny, mtx, dist)
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(img)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(top_down)
ax2.set_title('Undistorted and Warped Image', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
"""
These steps will all be useful when you get to the project at the end of this section.
First off, we defined a function for you, corners_unwarp(). The function accepts an image and our previously calculated values for the camera matrix, mtx, and distortion coefficients, dist.
Then you had to implement the function.
Here's how I implemented mine:
"""
# Define a function that takes an image, number of x and y points,
# camera matrix and distortion coefficients
def corners_unwarp(img, nx, ny, mtx, dist):
# Use the OpenCV undistort() function to remove distortion
undist = cv2.undistort(img, mtx, dist, None, mtx)
# Convert undistorted image to grayscale
gray = cv2.cvtColor(undist, cv2.COLOR_BGR2GRAY)
# Search for corners in the grayscaled image
ret, corners = cv2.findChessboardCorners(gray, (nx, ny), None)
if ret == True:
# If we found corners, draw them! (just for fun)
cv2.drawChessboardCorners(undist, (nx, ny), corners, ret)
# Choose offset from image corners to plot detected corners
# This should be chosen to present the result at the proper aspect ratio
# My choice of 100 pixels is not exact, but close enough for our purpose here
offset = 100 # offset for dst points
# Grab the image shape
img_size = (gray.shape[1], gray.shape[0])
# For source points I'm grabbing the outer four detected corners
src = np.float32([corners[0], corners[nx-1], corners[-1], corners[-nx]])
# For destination points, I'm arbitrarily choosing some points to be
# a nice fit for displaying our warped result
# again, not exact, but close enough for our purposes
dst = np.float32([[offset, offset], [img_size[0]-offset, offset],
[img_size[0]-offset, img_size[1]-offset],
[offset, img_size[1]-offset]])
# Given src and dst points, calculate the perspective transform matrix
M = cv2.getPerspectiveTransform(src, dst)
# Warp the image using OpenCV warpPerspective()
warped = cv2.warpPerspective(undist, M, img_size)
# Return the resulting image and matrix
return warped, M
"""
Pass in img and set the parameter orient as 'x' or 'y' to take either the x or y gradient. Set min_thresh, and max_thresh to specify the range to select for binary output. You can use exclusive (<, >) or inclusive (<=, >=) thresholding.
NOTE: Your output should be an array of the same size as the input image. The output array elements should be 1 where gradients were in the threshold range, and 0 everywhere else. As usual, if you run into any errors as you run your code, please refer to the Examples of Useful Code section in the previous video and make sure that your code syntax matches up!
"""
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import pickle
# Read in an image and grayscale it
image = mpimg.imread('signs_vehicles_xygrad.png')
# Define a function that applies Sobel x or y,
# then takes an absolute value and applies a threshold.
# Note: calling your function with orient='x', thresh_min=5, thresh_max=100
# should produce output like the example image shown above this quiz.
def abs_sobel_thresh(img, orient='x', thresh_min=0, thresh_max=255):
# Apply the following steps to img
# 1) Convert to grayscale
# 2) Take the derivative in x or y given orient = 'x' or 'y'
# 3) Take the absolute value of the derivative or gradient
# 4) Scale to 8-bit (0 - 255) then convert to type = np.uint8
# 5) Create a mask of 1's where the scaled gradient magnitude
# is > thresh_min and < thresh_max
# 6) Return this mask as your binary_output image
binary_output = np.copy(img) # Remove this line
return binary_output
# Run the function
grad_binary = abs_sobel_thresh(image, orient='x', thresh_min=20, thresh_max=100)
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(grad_binary, cmap='gray')
ax2.set_title('Thresholded Gradient', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
"""
# Define a function that takes an image, gradient orientation,
# and threshold min / max values.
def abs_sobel_thresh(img, orient='x', thresh_min=0, thresh_max=255):
# Convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Apply x or y gradient with the OpenCV Sobel() function
# and take the absolute value
if orient == 'x':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0))
if orient == 'y':
abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1))
# Rescale back to 8 bit integer
scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
# Create a copy and apply the threshold
binary_output = np.zeros_like(scaled_sobel)
# Here I'm using inclusive (>=, <=) thresholds, but exclusive is ok too
binary_output[(scaled_sobel >= thresh_min) & (scaled_sobel <= thresh_max)] = 1
# Return the result
return binary_output
"""
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import pickle
# Read in an image
image = mpimg.imread('signs_vehicles_xygrad.png')
# Define a function that applies Sobel x and y,
# then computes the magnitude of the gradient
# and applies a threshold
def mag_thresh(img, sobel_kernel=3, mag_thresh=(0, 255)):
# Apply the following steps to img
# 1) Convert to grayscale
# 2) Take the gradient in x and y separately
# 3) Calculate the magnitude
# 4) Scale to 8-bit (0 - 255) and convert to type = np.uint8
# 5) Create a binary mask where mag thresholds are met
# 6) Return this mask as your binary_output image
binary_output = np.copy(img) # Remove this line
return binary_output
# Run the function
mag_binary = mag_thresh(image, sobel_kernel=3, mag_thresh=(30, 100))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(mag_binary, cmap='gray')
ax2.set_title('Thresholded Magnitude', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
"""
# Define a function to return the magnitude of the gradient
# for a given sobel kernel size and threshold values
def mag_thresh(img, sobel_kernel=3, mag_thresh=(0, 255)):
# Convert to grayscale
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Take both Sobel x and y gradients
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
# Calculate the gradient magnitude
gradmag = np.sqrt(sobelx**2 + sobely**2)
# Rescale to 8 bit
scale_factor = np.max(gradmag)/255
gradmag = (gradmag/scale_factor).astype(np.uint8)
# Create a binary image of ones where threshold is met, zeros otherwise
binary_output = np.zeros_like(gradmag)
binary_output[(gradmag >= mag_thresh[0]) & (gradmag <= mag_thresh[1])] = 1
# Return the binary image
return binary_output
test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
test_output = mag_thresh(test_img)
show_two_images(test_img, test_output, title2="Magnitude Image")
When you play around with the thresholding for the gradient magnitude in the previous exercise, you find what you might expect, namely, that it picks up the lane lines well, but with a lot of other stuff detected too. Gradient magnitude is at the heart of Canny edge detection, and is why Canny works well for picking up all edges.
In the case of lane lines, we're interested only in edges of a particular orientation. So now we will explore the direction, or orientation, of the gradient.
The direction of the gradient is simply the inverse tangent (arctangent) of the y gradient divided by the x gradient:
arctan(sobel y /sobel x ).
Each pixel of the resulting image contains a value for the angle of the gradient away from horizontal in units of radians, covering a range of −π/2 to π/2. An orientation of 0 implies a horizontal line and orientations of +/−π/2 imply vertical lines.
In this next exercise, you'll write a function to compute the direction of the gradient and apply a threshold. The direction of the gradient is much noisier than the gradient magnitude, but you should find that you can pick out particular features by orientation.
Steps to take in this exercise: Fill out the function in the editor below to return a thresholded absolute value of the gradient direction. Use Boolean operators, again with exclusive (<, >) or inclusive (<=, >=) thresholds. Test that your function returns output similar to the example below for sobel_kernel=15, thresh=(0.7, 1.3).
"""
mport numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import pickle
# Read in an image
image = mpimg.imread('signs_vehicles_xygrad.png')
# Define a function that applies Sobel x and y,
# then computes the direction of the gradient
# and applies a threshold.
def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
# Apply the following steps to img
# 1) Convert to grayscale
# 2) Take the gradient in x and y separately
# 3) Take the absolute value of the x and y gradients
# 4) Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient
# 5) Create a binary mask where direction thresholds are met
# 6) Return this mask as your binary_output image
binary_output = np.copy(img) # Remove this line
return binary_output
# Run the function
dir_binary = dir_threshold(image, sobel_kernel=15, thresh=(0.7, 1.3))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(image)
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(dir_binary, cmap='gray')
ax2.set_title('Thresholded Grad. Dir.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)
"""
# Define a function to threshold an image for a given range and Sobel kernel
"""
def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
# Grayscale
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Calculate the x and y gradients
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
# Take the absolute value of the gradient direction,
# apply a threshold, and create a binary image result
absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
binary_output = np.zeros_like(absgraddir)
binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1
# Return the binary image
return binary_output
"""
If you play around with the thresholds in the last exercise, you'll find that you can start to identify the lane lines by gradient direction alone by setting the threshold around thresh = (0.7, 1.3), but there's still a lot of noise in the resulting image.
Now consider how you can use various aspects of your gradient measurements (x, y, magnitude, direction) to isolate lane-line pixels. Specifically, think about how you can use thresholds of the x and y gradients, the overall gradient magnitude, and the gradient direction to focus on pixels that are likely to be part of the lane lines. Challenge: In the project at the end of this section, you'll want to experiment with thresholding various aspects of the gradient, so now would be a great time to start coding it up on your local machine! Grab the image we've been working with for the last three quizzes here (or a smaller jpg file here).
Combine the selection thresholds from the last 3 quizzes to write a piece of code like the following, where you can play with various thresholds and see the output.
def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0, 255)):
# Calculate directional gradient
# Apply threshold
return grad_binary
def mag_thresh(image, sobel_kernel=3, mag_thresh=(0, 255)):
# Calculate gradient magnitude
# Apply threshold
return mag_binary
def dir_threshold(image, sobel_kernel=3, thresh=(0, np.pi/2)):
# Calculate gradient direction
# Apply threshold
return dir_binary
# Choose a Sobel kernel size
ksize = 3 # Choose a larger odd number to smooth gradient measurements
# Apply each of the thresholding functions
#gradx = abs_sobel_thresh(image, orient='x', sobel_kernel=ksize, thresh=(0, 255))
#grady = abs_sobel_thresh(image, orient='y', sobel_kernel=ksize, thresh=(0, 255))
#mag_binary = mag_thresh(image, sobel_kernel=ksize, mag_thresh=(0, 255))
#dir_binary = dir_threshold(image, sobel_kernel=ksize, thresh=(0, np.pi/2))
Try different combinations and see what you get.
For example, here is a selection for pixels where both the x and y gradients meet the threshold criteria, or the gradient magnitude and direction are both within their threshold values.
#combined = np.zeros_like(dir_binary)
#combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
Now it's time to combine what you know about color and gradient thresholding to get the best of both worlds. Here's an example of how that might look: At this point, it's okay to detect edges around trees or cars because these lines can be mostly filtered out by applying a mask to the image and essentially cropping out the area outside of the lane lines. It's most important that you reliably detect different colors of lane lines under varying degrees of daylight and shadow.
You can clearly see which parts of the lane lines were detected by the gradient threshold and which parts were detected by the color threshold by stacking the channels and seeing the individual components. You can create a binary combination of these two images to map out where either the color or gradient thresholds were met. Here's what that looks like in code:
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
# Edit this function to create your own pipeline.
def pipeline_thresh(img, s_thresh=(170, 255), sx_thresh=(20, 100)):
img = np.copy(img)
# Convert to HLS color space and separate the V channel
hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS).astype(np.float)
l_channel = hls[:,:,1]
s_channel = hls[:,:,2]
# Grayscale image
# NOTE: we already saw that standard grayscaling lost color information for the lane lines
# Explore gradients in other colors spaces / color channels to see what might work better
gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
# Sobel x
#sobelx = cv2.Sobel(l_channel, cv2.CV_64F, 1, 0) # Take the derivative in x
sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0) # Take the derivative in x
abs_sobelx = np.absolute(sobelx) # Absolute x derivative to accentuate lines away from horizontal
scaled_sobel = np.uint8(255*abs_sobelx/np.max(abs_sobelx))
# Threshold x gradient
sxbinary = np.zeros_like(scaled_sobel)
sxbinary[(scaled_sobel >= sx_thresh[0]) & (scaled_sobel <= sx_thresh[1])] = 1
# Threshold color channel
s_binary = np.zeros_like(s_channel)
s_binary[(s_channel >= s_thresh[0]) & (s_channel <= s_thresh[1])] = 1
# Stack each channel
# Note color_binary[:, :, 0] is all 0s, effectively an all black image. It might
# be beneficial to replace this channel with something else.
color_binary = np.dstack(( np.zeros_like(sxbinary), sxbinary, s_binary)) * 255
# Combine the two binary thresholds
combined_binary = np.zeros_like(sxbinary)
combined_binary[(s_binary == 1) | (sxbinary == 1)] = 1
#combined_binary[(s_binary == 1)] = 1
return combined_binary
#return color_binary
#return gray
#image = mpimg.imread('./test_images/test5.jpg')
test_image = mpimg.imread('./test_images/straight_lines1.jpg')
image_threshed = pipeline_thresh(test_image)
show_two_images(test_image, image_threshed, title2="Thresh Pipeline Result")
test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
test_output = pipeline_thresh(test_img, s_thresh=(170, 255), sx_thresh=(5, 100))
show_two_images(test_img, test_output, title2="Magnitude Image")
undistorted = cal_undistort(test_image, objpoints, imgpoints)
show_two_images(test_image, undistorted, title2="Undistorted Image")
Y_TOP = 435#440
Y_BTM = 668
X_CTR = 640#640 = 1280/2
X_TOP_WDT = 48#45#50#55#65#50#50#60#60
X_BTM_WDT = 1000#1280#1000#1000#900#850
# less space is used
Y_TOP = 450#435#440
X_TOP_WDT = 120#48#45#50#55#65#50#50#60#60
TOP_LEFT = (X_CTR - X_TOP_WDT/2 , Y_TOP)
TOP_RIGHT = (X_CTR + X_TOP_WDT/2, Y_TOP)
BTM_LEFT = (X_CTR - X_BTM_WDT/2 , Y_BTM)
BTM_RIGHT = (X_CTR + X_BTM_WDT/2, Y_BTM)
SRC_TRAPEZOID_ORG_ORDR = [TOP_LEFT, TOP_RIGHT, BTM_RIGHT, BTM_LEFT]
#SRC_TRAPEZOID_WRP = [TOP_LEFT, TOP_RIGHT, BTM_LEFT, BTM_RIGHT]
def pipeline_warp(image, offsetX = 50, offsetY = 0):
offset = 100 # offset for dst points
# Grab the image shape
img_size = (image.shape[1], image.shape[0])
w = img_size[1]
h = img_size[1]
offsetX = 30 # test
src = np.float32(SRC_TRAPEZOID_ORG_ORDR)#SRC_RECTANGLE)
#error: ..\..\..\modules\imgproc\src\imgwarp.cpp:6101: error: (-215) _src.total() > 0 in function cv::warpPerspective
#src = np.float32([corners[0],corners[7],corners[40],corners[47]])#([[,],[,],[,],[,]])
# c) define 4 destination points dst = np.float32([[,],[,],[,],[,]])
# For destination points, I'm arbitrarily choosing some points to be
# a nice fit for displaying our warped result
# again, not exact, but close enough for our purposes
#dst = np.float32([[offset, offset], [img_size[0]-offset, offset],
# [img_size[0]-offset, img_size[1]-offset],
# [offset, img_size[1]-offset]])
#dst = np.float32([[offsetX, offsetY], [w - offsetX, offsetY],
# [offsetX, h - offsetY], [w - offsetX, h - offsetY]])
dst = np.float32([[offsetX, offsetY], [w - offsetX, offsetY],
[w - offsetX, h - offsetY],
[offsetX, h - offsetY]])
#dst = np.float32([[,],[,],[,],[,]])
# d) use cv2.getPerspectiveTransform() to get M, the transform matrix
M = cv2.getPerspectiveTransform(src, dst)
# e) use cv2.warpPerspective() to warp your image to a top-down view
#warped = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
warped = cv2.warpPerspective(image,M,(h, w))
Minv = cv2.getPerspectiveTransform(dst, src)
return warped, M, Minv
#test_img = mpimg.imread('./test_images/straight_lines1.jpg')
test_img = mpimg.imread('./test_images/test1.jpg')
image_threshed = pipeline_thresh(test_img)
image_warped, M, Minv = pipeline_warp(image_threshed)
show_two_images(test_img, image_threshed, title2="Binary Image")
show_two_images(test_img, image_warped, title2="Warped Image")
After you've tuned your pipeline on test images, you'll run on a video stream, just like in the first project. In this case, however, you're going to keep track of things like where your last several detections of the lane lines were and what the curvature was, so you can properly treat new detections. To do this, it's useful to define a Line() class to keep track of all the interesting parameters you measure from frame to frame. Here's an example:
You can create an instance of the Line() class for the left and right lane lines to keep track of recent detections and to perform sanity checks.
# Define a class to receive the characteristics of each line detection
class Line():
def __init__(self):
# was the line detected in the last iteration?
self.detected = False
# x values of the last n fits of the line
self.recent_xfitted = []
#average x values of the fitted line over the last n iterations
self.bestx = None
#polynomial coefficients averaged over the last n iterations
self.best_fit = None
#polynomial coefficients for the most recent fit
self.current_fit = [np.array([False])]
self.prev_fit = [np.array([False])]
#radius of curvature of the line in some units
self.radius_of_curvature = None
#distance in meters of vehicle center from the line
self.line_base_pos = None
#difference in fit coefficients between last and new fits
self.diffs = np.array([0,0,0], dtype='float')
#x values for detected line pixels
self.allx = None
#y values for detected line pixels
self.ally = None
def calc_diffs():
self.diffs[0] = self.current_fit[0] - self.prev_fit[0]
self.diffs[1] = self.current_fit[1] - self.prev_fit[1]
self.diffs[2] = self.current_fit[2] - self.prev_fit[2]
def update_average_and_best(self):
#average x values of the fitted line over the last n iterations
self.bestx = None
#polynomial coefficients averaged over the last n iterations
self.best_fit = None
def compare_last_and_recent(self):
#difference in fit coefficients between last and new fits
self.diffs = np.array([0,0,0], dtype='float')
def add_recent_data(self, current_fit, detected_x, detected_y):
# was the line detected in the last iteration?
#self.detected = detected
#polynomial coefficients for the most recent fit
self.current_fit = current_fit#[np.array([False])]
# x values of the last n fits of the line
self.recent_xfitted.append(current_fit)# = [] #?
#x values for detected line pixels
self.allx = detected_x #? None
#y values for detected line pixels
self.ally = detected_y #? None
# Assuming you have created a warped binary image called "binary_warped"
def find_lines(binary_warped):
# Choose the number of sliding windows
nwindows = 9
# Set the width of the windows +/- margin
margin = 100
#margin = 150
# Set minimum number of pixels found to recenter window
minpix = 50
# Take a histogram of the bottom half of the image
#histogram = np.sum(binary_warped[binary_warped.shape[0]/2:,:], axis=0)
histogram = np.sum(binary_warped[np.int(binary_warped.shape[0]/2):,:], axis=0)
# Create an output image to draw on and visualize the result
out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
# Find the peak of the left and right halves of the histogram
# These will be the starting point for the left and right lines
midpoint = np.int(histogram.shape[0]/2)
leftx_base = np.argmax(histogram[:midpoint])
rightx_base = np.argmax(histogram[midpoint:]) + midpoint
# Set height of windows
window_height = np.int(binary_warped.shape[0]/nwindows)
# Identify the x and y positions of all nonzero pixels in the image
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
# Current positions to be updated for each window
leftx_current = leftx_base
rightx_current = rightx_base
# Create empty lists to receive left and right lane pixel indices
left_lane_inds = []
right_lane_inds = []
# Step through the windows one by one
right_stop = False
left_stop = False
for window in range(nwindows):
# Identify window boundaries in x and y (and right and left)
win_y_low = binary_warped.shape[0] - (window+1)*window_height
win_y_high = binary_warped.shape[0] - window*window_height
win_xleft_low = leftx_current - margin
win_xleft_high = leftx_current + margin
win_xright_low = rightx_current - margin
win_xright_high = rightx_current + margin
# Draw the windows on the visualization image
cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),
(0,255,0), 2)
cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),
#(0,255,0), 2)
(0,0,255), 2)
# Identify the nonzero pixels in x and y within the window
good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
(nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) &
(nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
# Append these indices to the lists
##if left_stop == False:
left_lane_inds.append(good_left_inds)
##if right_stop == False:
right_lane_inds.append(good_right_inds)
# If you found > minpix pixels, recenter next window on their mean position
if len(good_left_inds) > minpix:
leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
if len(good_right_inds) > minpix:
rightx_current = np.int(np.mean(nonzerox[good_right_inds]))
# If
##if (len(nonzerox[good_left_inds]) < 1 and window > nwindows//2) :
## left_stop = True
##if (len(nonzerox[good_right_inds]) < 1 and window > nwindows//2) :
## right_stop = True
#print(len(nonzerox[good_left_inds]), nwindows//2, window,( window > nwindows//2) )
# Concatenate the arrays of indices
left_lane_inds = np.concatenate(left_lane_inds)
right_lane_inds = np.concatenate(right_lane_inds)
# Extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
right_line = Line()
left_line = Line()
right_line.update_average_and_best()
right_line.add_recent_data(right_fit, rightx, righty)
left_line.add_recent_data(left_fit, leftx, lefty)
# was the line detected in the last iteration?
right_line.detected = False
# x values of the last n fits of the line
right_line.recent_xfitted = []
#average x values of the fitted line over the last n iterations
right_line.bestx = None
#polynomial coefficients averaged over the last n iterations
right_line.best_fit = None
#polynomial coefficients for the most recent fit
right_line.current_fit = right_fit #[np.array([False])]
#radius of curvature of the line in some units
right_line.radius_of_curvature = None
#distance in meters of vehicle center from the line
right_line.line_base_pos = None
#difference in fit coefficients between last and new fits
right_line.diffs = np.array([0,0,0], dtype='float')
return left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img
Now you know where the lines are you have a fit! In the next frame of video you don't need to do a blind search again, but instead you can just search in a margin around the previous line position like this:
# Assume you now have a new warped binary image
# from the next frame of video (also called "binary_warped")
# It's now much easier to find line pixels!
def find_lines_again(binary_warped, left_fit, right_fit):
nonzero = binary_warped.nonzero()
nonzeroy = np.array(nonzero[0])
nonzerox = np.array(nonzero[1])
margin = 100
margin = 50
#margin = 30
left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy +
left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) +
left_fit[1]*nonzeroy + left_fit[2] + margin)))
right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy +
right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) +
right_fit[1]*nonzeroy + right_fit[2] + margin)))
# Again, extract left and right line pixel positions
leftx = nonzerox[left_lane_inds]
lefty = nonzeroy[left_lane_inds]
rightx = nonzerox[right_lane_inds]
righty = nonzeroy[right_lane_inds]
# Fit a second order polynomial to each
left_fit = np.polyfit(lefty, leftx, 2)
right_fit = np.polyfit(righty, rightx, 2)
# Generate x and y values for plotting
ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
return left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img
# Calculate the new radii of curvature
def calc_radii_of_curvature(fit_cr, y_eval, ym_per_pix):
curverad = ((1 + (2*fit_cr[0]*y_eval*ym_per_pix + fit_cr[1])**2)**1.5) / np.absolute(2*fit_cr[0])
return curverad
#left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
#right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
import numpy as np
import matplotlib.pyplot as plt
# Generate some fake data to represent lane-line pixels
ploty = np.linspace(0, 719, num=720)# to cover same y-range as image
quadratic_coeff = 3e-4 # arbitrary quadratic coefficient
# For each y position generate random x position within +/-50 pix
# of the line base position in each case (x=200 for left, and x=900 for right)
leftx = np.array([200 + (y**2)*quadratic_coeff + np.random.randint(-50, high=51)
for y in ploty])
rightx = np.array([900 + (y**2)*quadratic_coeff + np.random.randint(-50, high=51)
for y in ploty])
leftx = leftx[::-1] # Reverse to match top-to-bottom in y
rightx = rightx[::-1] # Reverse to match top-to-bottom in y
# Fit a second order polynomial to pixel positions in each fake lane line
left_fit = np.polyfit(ploty, leftx, 2)
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fit = np.polyfit(ploty, rightx, 2)
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
# Plot up the fake data
mark_size = 3
plt.plot(leftx, ploty, 'o', color='red', markersize=mark_size)
plt.plot(rightx, ploty, 'o', color='blue', markersize=mark_size)
plt.xlim(0, 1280)
plt.ylim(0, 720)
plt.plot(left_fitx, ploty, color='green', linewidth=3)
plt.plot(right_fitx, ploty, color='green', linewidth=3)
plt.gca().invert_yaxis() # to visualize as we do the images
#print("left:", calc_radii_of_curvature(left_fit, y_eval, ym_per_pix))
#print("riht:",calc_radii_of_curvature(right_fit, y_eval, ym_per_pix))
# Define y-value where we want radius of curvature
# I'll choose the maximum y-value, corresponding to the bottom of the image
y_eval = np.max(ploty)
left_curverad = ((1 + (2*left_fit[0]*y_eval + left_fit[1])**2)**1.5) / np.absolute(2*left_fit[0])
right_curverad = ((1 + (2*right_fit[0]*y_eval + right_fit[1])**2)**1.5) / np.absolute(2*right_fit[0])
print(left_curverad, right_curverad)
# Example values: 1926.74 1908.48
# Define y-value where we want radius of curvature
# I'll choose the maximum y-value, corresponding to the bottom of the image
def calc_curverad(ploty, fit):
y_eval = np.max(ploty)
curverad = ((1 + (2*fit[0]*y_eval + fit[1])**2)**1.5) / np.absolute(2*fit[0])
return curverad
# Example values: 1926.74 1908.48
# Define conversions in x and y from pixels space to meters
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
# Fit new polynomials to x,y in world space
left_fit_cr = np.polyfit(ploty*ym_per_pix, leftx*xm_per_pix, 2)
right_fit_cr = np.polyfit(ploty*ym_per_pix, rightx*xm_per_pix, 2)
# Calculate the new radii of curvature
left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
# Now our radius of curvature is in meters
print(left_curverad, 'm', right_curverad, 'm')
# Example values: 632.1 m 626.2 m
def calc_radius_in_meter(y_eval, left_fit_cr, right_fit_cr):
# Define conversions in x and y from pixels space to meters
ym_per_pix = 30/720 # meters per pixel in y dimension
xm_per_pix = 3.7/700 # meters per pixel in x dimension
# Calculate the new radii of curvature
left_curverad = ((1 + (2*left_fit_cr[0]*y_eval*ym_per_pix + left_fit_cr[1])**2)**1.5) / np.absolute(2*left_fit_cr[0])
right_curverad = ((1 + (2*right_fit_cr[0]*y_eval*ym_per_pix + right_fit_cr[1])**2)**1.5) / np.absolute(2*right_fit_cr[0])
# Now our radius of curvature is in meters
return left_curverad, right_curverad
# Example values: 632.1 m 626.2 m
def calc_vehicle_position(left_fit, right_fit, image):
#y_btm = 720#
y_btm = image.shape[0] # Y
#print("y_btm", y_btm)
#image_mid = 720//2
image_mid = image.shape[1]/2 # X
#print("image_mid", image_mid)
x_left = left_fit[0]*y_btm**2 + left_fit[1]*y_btm + left_fit[2]
x_right = right_fit[0]*y_btm**2 + right_fit[1]*y_btm + right_fit[2]
#print("x_left", x_left)
#print("x_right", x_right)
lane_width = x_right - x_left
x_mid = x_left + (lane_width)//2
#print("x_mid", x_mid)
vehicle_position = image_mid - x_mid
LANE_WIDTH_METER = 3.4
meter_per_pixel = LANE_WIDTH_METER/lane_width
vehicle_position_meter = vehicle_position*meter_per_pixel
return vehicle_position_meter#vehicle_position
def put_text(image, text, margin_from_edge = 5, font = cv2.FONT_HERSHEY_PLAIN, font_size = 0.6, color = (255,255,0)):
#w = image.shape[0]
#h = image.shape[1]
start_x = margin_from_edge
start_y = margin_from_edge
cv2.putText(image,text,(start_x,start_y),font, font_size,color)
At this point, you're done! But here is how you can visualize the result as well:
def draw_rectangle(img):
#result = cv.rectangle(img, (10, 50), (50, 150), (255, 0, 0), 3, 4)
#image = cv2.polylines(img, np.float32(SRC_RECTANGLE), True, (0, 0, 255), 5)
pts = np.array(SRC_TRAPEZOID_ORG_ORDR, np.int32)
pts = pts.reshape((-1, 1, 2))
#pts = np.float32(SRC_RECTANGLE)
copy = img.copy()
cv2.polylines(copy, [pts],True,(255,0,0), thickness=2)
return copy
#X_CENTER = 410
#Y_CENTER = 610
#X_BOTTOM = 660
#X_OFFSET = 5
#Y_OFFSET = 400
class LineProjector():
def __init__(self, Minv):
self.Minv = Minv
def project_lines(self, original_image, warped, left_fit, right_fit):
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
# Create an image to draw the lines on
warp_zero = np.zeros_like(warped).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
# Recast the x and y points into usable format for cv2.fillPoly()
pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
pts = np.hstack((pts_left, pts_right))
# Draw the lane onto the warped blank image
cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))
# Warp the blank back to original image space using inverse perspective matrix (Minv)
#newwarp = cv2.warpPerspective(color_warp, self.Minv, (image.shape[1], image.shape[0]))
newwarp = cv2.warpPerspective(color_warp, self.Minv, (original_image.shape[1], original_image.shape[0]))
# Combine the result with the original image
result = cv2.addWeighted(original_image, 1, newwarp, 0.3, 0)
return result
# Generate x and y values for plotting
def visualize_lines(warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img):
ploty = np.linspace(0, warped.shape[0]-1, warped.shape[0] )
out_img[nonzeroy[left_lane_inds], nonzerox[left_lane_inds]] = [255, 0, 0]
out_img[nonzeroy[right_lane_inds], nonzerox[right_lane_inds]] = [0, 0, 255]
plt.imshow(out_img)
# Generate x and y values for plotting
left_fitx = left_fit[0]*ploty**2 + left_fit[1]*ploty + left_fit[2]
right_fitx = right_fit[0]*ploty**2 + right_fit[1]*ploty + right_fit[2]
plt.plot(left_fitx, ploty, color='yellow')
plt.plot(right_fitx, ploty, color='yellow')
#print(warped.shape[0], warped.shape[1])
plt.xlim(0, 1280)
plt.ylim(720, 0)
test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)
#print("left:", calc_radii_of_curvature(left_fit, y_eval, ym_per_pix))
#print("riht:",calc_radii_of_curvature(right_fit, y_eval, ym_per_pix))
test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit, right_fit)
visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)
Another way to approach the sliding window method is to apply a convolution, which will maximize the number of "hot" pixels in each window. A convolution is the summation of the product of two separate signals, in our case the window template and the vertical slice of the pixel image.
You slide your window template across the image from left to right and any overlapping values are summed together, creating the convolved signal. The peak of the convolved signal is where there was the highest overlap of pixels and the most likely position for the lane marker.
Now let's try using convolutions to find the best window center positions in a thresholded road image. The code below allows you to experiment with using convolutions for a sliding window search function. Go ahead and give it a try.
def window_mask(width, height, img_ref, center,level):
output = np.zeros_like(img_ref)
output[int(img_ref.shape[0]-(level+1)*height):int(img_ref.shape[0]-level*height),max(0,int(center-width/2)):min(int(center+width/2),img_ref.shape[1])] = 1
return output
def find_window_centroids(image, window_width, window_height, margin):
window_centroids = [] # Store the (left,right) window centroid positions per level
window = np.ones(window_width) # Create our window template that we will use for convolutions
# First find the two starting positions for the left and right lane by using np.sum to get the vertical image slice
# and then np.convolve the vertical image slice with the window template
# Sum quarter bottom of image to get slice, could use a different ratio
l_sum = np.sum(image[int(3*image.shape[0]/4):,:int(image.shape[1]/2)], axis=0)
l_center = np.argmax(np.convolve(window,l_sum))-window_width/2
r_sum = np.sum(image[int(3*image.shape[0]/4):,int(image.shape[1]/2):], axis=0)
r_center = np.argmax(np.convolve(window,r_sum))-window_width/2+int(image.shape[1]/2)
# Add what we found for the first layer
window_centroids.append((l_center,r_center))
# Go through each layer looking for max pixel locations
for level in range(1,(int)(image.shape[0]/window_height)):
# convolve the window into the vertical slice of the image
image_layer = np.sum(image[int(image.shape[0]-(level+1)*window_height):int(image.shape[0]-level*window_height),:], axis=0)
conv_signal = np.convolve(window, image_layer)
# Find the best left centroid by using past left center as a reference
# Use window_width/2 as offset because convolution signal reference is at right side of window, not center of window
offset = window_width/2
l_min_index = int(max(l_center+offset-margin,0))
l_max_index = int(min(l_center+offset+margin,image.shape[1]))
l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset
# Find the best right centroid by using past right center as a reference
r_min_index = int(max(r_center+offset-margin,0))
r_max_index = int(min(r_center+offset+margin,image.shape[1]))
r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset
# Add what we found for that layer
window_centroids.append((l_center,r_center))
return window_centroids
def gen_output(window_centroids, warped):
# If we found any window centers
output = None
if len(window_centroids) > 0:
# Points used to draw all the left and right windows
l_points = np.zeros_like(warped)
r_points = np.zeros_like(warped)
# Go through each level and draw the windows
for level in range(0,len(window_centroids)):
# Window_mask is a function to draw window areas
l_mask = window_mask(window_width,window_height,warped,window_centroids[level][0],level)
r_mask = window_mask(window_width,window_height,warped,window_centroids[level][1],level)
# Add graphic points from window mask here to total pixels found
l_points[(l_points == 255) | ((l_mask == 1) ) ] = 255
r_points[(r_points == 255) | ((r_mask == 1) ) ] = 125#255
# Draw the results
template = np.array(r_points+l_points,np.uint8) # add both left and right window pixels together
zero_channel = np.zeros_like(template) # create a zero color channel
template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) # make window pixels green
warpage= np.dstack((warped, warped, warped))*255 # making the original road pixels 3 color channels
output = cv2.addWeighted(warpage, 1, template, 0.5, 0.0) # overlay the orignal road image with window results
# If no window centers found, just display orginal road image
else:
output = np.array(cv2.merge((warped,warped,warped)),np.uint8)
return output
# Read in a thresholded image
#warped = mpimg.imread('warped_example.jpg')
test_img = mpimg.imread('./test_images/test2.jpg')
image_threshed = pipeline_thresh(test_img)
binary_warped, M, Minv = pipeline_warp(image_threshed)
# window settings
window_width = 50
window_height = 80 # Break image into 9 vertical layers since image height is 720
margin = 100 # How much to slide left and right for searching
window_centroids = find_window_centroids(binary_warped, window_width, window_height, margin)
output = gen_output(window_centroids, binary_warped)
# Display the final results
plt.imshow(output)
plt.title('window fitting results')
plt.show()
"""
# If we found any window centers
if len(window_centroids) > 0:
# Points used to draw all the left and right windows
l_points = np.zeros_like(warped)
r_points = np.zeros_like(warped)
# Go through each level and draw the windows
for level in range(0,len(window_centroids)):
# Window_mask is a function to draw window areas
l_mask = window_mask(window_width,window_height,warped,window_centroids[level][0],level)
r_mask = window_mask(window_width,window_height,warped,window_centroids[level][1],level)
# Add graphic points from window mask here to total pixels found
l_points[(l_points == 255) | ((l_mask == 1) ) ] = 255
r_points[(r_points == 255) | ((r_mask == 1) ) ] = 255
# Draw the results
template = np.array(r_points+l_points,np.uint8) # add both left and right window pixels together
zero_channel = np.zeros_like(template) # create a zero color channel
template = np.array(cv2.merge((zero_channel,template,zero_channel)),np.uint8) # make window pixels green
warpage= np.dstack((warped, warped, warped))*255 # making the original road pixels 3 color channels
output = cv2.addWeighted(warpage, 1, template, 0.5, 0.0) # overlay the orignal road image with window results
# If no window centers found, just display orginal road image
else:
output = np.array(cv2.merge((warped,warped,warped)),np.uint8)
"""
Y_TOP = 450#435#440
Y_BTM = 668
X_CTR = 640#640 = 1280/2
X_TOP_WDT = 120#48#45#50#55#65#50#50#60#60
X_BTM_WDT = 1000#1280#1000#1000#900#850
Y_TOP = 460#435#440
X_TOP_WDT = 150#48#45#50#55#65#50#50#60#60
#X_BTM_WDT = 900
TOP_LEFT = (X_CTR - X_TOP_WDT/2 , Y_TOP)
TOP_RIGHT = (X_CTR + X_TOP_WDT/2, Y_TOP)
BTM_LEFT = (X_CTR - X_BTM_WDT/2 , Y_BTM)
BTM_RIGHT = (X_CTR + X_BTM_WDT/2, Y_BTM)
SRC_TRAPEZOID_ORG_ORDR = [TOP_LEFT, TOP_RIGHT, BTM_RIGHT, BTM_LEFT]
#SRC_TRAPEZOID_WRP = [TOP_LEFT, TOP_RIGHT, BTM_LEFT, BTM_RIGHT]
def show_process_images(image, objpoints, imgpoints, left_fit_prev=None, right_fit_prev=None, sx_thresh=(20, 100)):
#put_text(image, "test")
#cv2.putText(image,text,(start_x,start_y),font, font_size,color)
image = cv2.putText(image,"text",(10,10),fontFace = cv2.FONT_HERSHEY_PLAIN, fontScale = 10, color = (255,255,0))
undistorted = cal_undistort(image, objpoints, imgpoints)
#show_two_images(image, undistorted)
thresed = pipeline_thresh(undistorted, sx_thresh=sx_thresh)
#show_two_images(undistorted, thresed)
binary_warped, M, Minv = pipeline_warp(thresed)
undistorted_w_line = draw_rectangle(undistorted)
#show_two_images(undistorted_w_line, binary_warped)
if not (left_fit_prev==None and right_fit_prev==None):
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit_prev, right_fit_prev)
else:
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
line_projector = LineProjector(Minv)
#visualize_lines(ploty, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy)
result = line_projector.project_lines(undistorted_w_line, binary_warped, left_fit, right_fit)
#result = project_lines(org_image, warped, left_fit, right_fit, ploty, Minv)
#plt.imshow(result)
show_two_images(thresed, binary_warped, title1="Original Image", title2="Warped Image")
empty = None
ploty = np.linspace(0, image.shape[0]-1, image.shape[0] )
y_eval = np.max(ploty)
left_curverad, right_curverad = calc_radius_in_meter(y_eval, left_fit, right_fit)
#print(left_curverad, right_curverad)
vehicle_position = calc_vehicle_position(left_fit, right_fit, binary_warped)
text_radius = "Radius:{} ".format((left_curverad + right_curverad)/2)
text_position = "Vehicle Position:{} ".format(vehicle_position)
#put_text(result, text)
print(text_position)
cv2.putText(result,text_radius,(50,100),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
cv2.putText(result,text_position,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,255,0))#255,255,255))
show_two_images(result, result)
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
visualize_lines(binary_warped, left_fit, right_fit, left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img)
return left_fit, right_fit
test_img = mpimg.imread('./test_images/straight_lines1.jpg')
#print(test_img.shape[0])
#print(test_img.shape[1])
show_process_images(test_img, objpoints, imgpoints)
test_img = mpimg.imread('./test_images/test1.jpg')
show_process_images(test_img, objpoints, imgpoints)
test_img = mpimg.imread('./test_images/test2.jpg')
show_process_images(test_img, objpoints, imgpoints)
test_img = mpimg.imread('./test_images/test4.jpg')
left_fit_prev, right_fit_prev = show_process_images(test_img, objpoints, imgpoints)
So you can see that your yellow lane detection is not strong at all, since you are able to detect yellows only very close to the car, where the clarity is much better. The approach to better this is to
Try other color spaces. For instance looking at the blue and green channels of the RGB might give you yellow pixels easily (yellow = blue + green right) Try to limit your search based on the expected lane width. This will prevent the weird effects that you see in your final image. In every detection, reinforce the weaker lane (lane with lesser pixels) with the stronger lane by assuming that the two lanes are parallel.
If there are too few pixels detected in a particular lane, just discard it and use the previous lane as the value. One another approach I took was to do a weighted average of the polynomials from previous frames to current one, based on the number of thresholded pixels. So if the current image has a lot of thresholded pixels, then I give the current polynomial a lot of importance, otherwise no. This is a generalization of point 3.
test_img = mpimg.imread('./issue_images/org_video_image_20171217_10_22_06.jpg')
#test_output = pipeline_thresh(test_img, s_thresh=(170, 255), sx_thresh=(5, 100))
show_process_images(test_img, objpoints, imgpoints)#, sx_thresh=(5, 100))
show_process_images(test_img, objpoints, imgpoints,left_fit_prev, right_fit_prev)
Once you have a good measurement of the line positions in warped space, it's time to project your measurement back down onto the road! Let's suppose, as in the previous example, you have a warped binary image called warped, and you have fit the lines with a polynomial and have arrays called ploty, left_fitx and right_fitx, which represent the x and y pixel values of the lines. You can then project those lines onto the original image as follows:
"""
org_image = undistorted
line_projector = LineProjector(Minv)
result = line_projector.project_lines(org_image, binary_warped, left_fit, right_fit)
#result = project_lines(org_image, warped, left_fit, right_fit, Minv)
plt.imshow(result)
"""
flg_in_process = False
left_fit = None
right_fit = None
count = 0
DIR_ORG_VIDEO_SHOT = "./video_images/"
DIR_CVT_VIDEO_SHOT = "./converted_images/"
ORG_VIDEO_FILE_NAME = "org_video_image_"
CVT_VIDEO_FILE_NAME = "cvt_video_image_"
from PIL import Image
def save_image(image, dirname, filename):
"""save a image file"""
filepath = dirname + filename + datetime.now().strftime("%Y%m%d_%H_%M_%S.jpg")
if not os.path.exists(filepath) :
Image.fromarray(image).save(filepath)
from datetime import datetime
import os
def video_pipeline(image):
#save_image(image,DIR_ORG_VIDEO_SHOT, ORG_VIDEO_FILE_NAME)
processed_image = np.copy(image)
# Undistort image (Try)
processed_image = cal_undistort(processed_image, objpoints, imgpoints)
processed_image = pipeline_thresh(processed_image)
binary_warped, M, Minv = pipeline_warp(processed_image)
if flg_in_process == False:
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines(binary_warped)
else:
left_fit, right_fit,left_lane_inds, right_lane_inds, nonzerox, nonzeroy, out_img = find_lines_again(binary_warped, left_fit, right_fit)
#ploty = np.linspace(0, binary_warped.shape[0]-1, binary_warped.shape[0] )
line_projector = LineProjector(Minv)
processed_image = line_projector.project_lines(image, binary_warped, left_fit, right_fit)#, ploty, Minv)
#ploty = np.linspace(0, image.shape[0]-1, image.shape[0] )
#y_eval = np.max(ploty)
#left_curverad, right_curverad = calc_radius_in_meter(y_eval, left_fit, right_fit)
#text = "Radius:{} ".format((left_curverad + right_curverad)/2)
#cv2.putText(processed_image,text,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
vehicle_position = calc_vehicle_position(left_fit, right_fit, binary_warped)
text_radius = "Radius:{} ".format((left_curverad + right_curverad)/2)
text_position = "Vehicle Position:{} ".format(vehicle_position)
cv2.putText(processed_image,text_radius,(50,100),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,0,255))#255,255,255))
cv2.putText(processed_image,text_position,(50,200),fontFace = cv2.FONT_ITALIC, fontScale = 3, color = (0,255,0))#255,255,255))
#save_image(processed_image,DIR_CVT_VIDEO_SHOT, CVT_VIDEO_FILE_NAME)
return processed_image # This must be a color image
def process_image(image):
# NOTE: The output you return should be a color image (3 channel) for processing video below
# TODO: put your pipeline here,
# you should return the final output (image where lines are drawn on lanes)
result = video_pipeline(image)
return result
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML
VIDEO_INPUT = 'challenge_video.mp4'
VIDEO_OUTPUT = 'output_images/challenge_video_output.mp4'
## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
## To do so add .subclip(start_second,end_second) to the end of the line below
## Where start_second and end_second are integer values representing the start and end of the subclip
## You may also uncomment the following line for a subclip of the first 5 seconds
##clip1 = VideoFileClip("test_videos/solidWhiteRight.mp4").subclip(0,5)
clip1 = VideoFileClip(VIDEO_INPUT)
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time white_clip.write_videofile(VIDEO_OUTPUT, audio=False)
flg_in_process = False
left_fit = None
right_fit = None
HTML("""
<video width="960" height="540" controls>
<source src="{0}">
</video>
""".format(VIDEO_OUTPUT))